@link-assistant/web-capture 1.7.26 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +35 -28
- package/bin/web-capture.js +148 -59
- package/package.json +3 -2
- package/src/archive.js +60 -6
- package/src/extract-images.js +90 -0
- package/src/markdown.js +9 -7
- package/tests/integration/api-endpoints.test.js +9 -0
- package/tests/integration/archive-default-layout.test.js +37 -0
- package/tests/unit/image-mode-defaults.test.js +54 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# @link-assistant/web-capture
|
|
2
2
|
|
|
3
|
+
## 1.8.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 09f83f3: Pin the default image-mode contract and route every capture path through a single image-handling chokepoint (`applyImageMode`), so the same flag behaves identically regardless of capture method (browser vs API, CLI vs server) — issue #112. Default `--format markdown` now references images by their direct remote URL (no `images/` folder, no inline base64); inline base64 (which has no remote URL to restore) is stripped to a visible placeholder with a warning instead of being silently kept as a multi-megabyte blob. `--embed-images` keeps base64 inline for a self-contained file. The new `--extract-images[=DIR]` flag extracts inline base64 **and** downloads remote images into `DIR/images/`, rewriting the markdown to reference the local files; on download failure the original remote URL is restored so references never break. `--keep-original-links` remains a back-compat alias for the default behavior.
|
|
8
|
+
|
|
9
|
+
## 1.7.27
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- 9cbb4c7: Pin the default `--format archive` layout contract so the zip always contains exactly `document.md` + `document.html` + `images/` (issue #113). The markdown-format archive endpoint previously bundled only `document.md` and `images/`, omitting the reference `document.html`; it now includes it on every path. Adds a reusable `buildArchiveFromHtml(html, baseUrl)` helper and an integration test that pins the layout so a future refactor cannot regress it.
|
|
14
|
+
|
|
3
15
|
## 1.7.26
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -73,11 +73,17 @@ web-capture https://example.com --archive
|
|
|
73
73
|
web-capture https://example.com --archive zip -o site.zip
|
|
74
74
|
web-capture https://example.com --archive tar.gz -o site.tar.gz
|
|
75
75
|
|
|
76
|
-
#
|
|
76
|
+
# Default markdown keeps remote image URLs as direct links
|
|
77
|
+
web-capture https://example.com -o page.md
|
|
78
|
+
|
|
79
|
+
# Keep images inline as base64 (self-contained file, opt-in)
|
|
77
80
|
web-capture https://example.com --embed-images -o page.md
|
|
78
81
|
|
|
82
|
+
# Extract images (inline base64 + remote) to a local images/ folder
|
|
83
|
+
web-capture https://example.com --extract-images -o page.md
|
|
84
|
+
|
|
79
85
|
# Custom images directory
|
|
80
|
-
web-capture https://example.com --images-dir assets -o page.md
|
|
86
|
+
web-capture https://example.com --extract-images --images-dir assets -o page.md
|
|
81
87
|
|
|
82
88
|
# Disable specific features
|
|
83
89
|
web-capture https://example.com --no-extract-latex --no-post-process -o page.md
|
|
@@ -245,32 +251,33 @@ web-capture --serve [--port <port>]
|
|
|
245
251
|
web-capture <url> [options]
|
|
246
252
|
```
|
|
247
253
|
|
|
248
|
-
| Option | Short | Description
|
|
249
|
-
| --------------------------- | ----- |
|
|
250
|
-
| `--format` | `-f` | Output format (see below)
|
|
251
|
-
| `--output` | `-o` | Output file path. Use `-o -` for stdout
|
|
252
|
-
| `--data-dir` | | Base directory for auto-derived output paths
|
|
253
|
-
| `--engine` | `-e` | Browser engine: `puppeteer`, `playwright`
|
|
254
|
-
| `--theme` | `-t` | Color scheme: `light`, `dark`, `no-preference`
|
|
255
|
-
| `--width` | | Viewport width in pixels
|
|
256
|
-
| `--height` | | Viewport height in pixels
|
|
257
|
-
| `--quality` | | JPEG quality 0-100
|
|
258
|
-
| `--fullPage` | | Capture full scrollable page
|
|
259
|
-
| `--embed-images` | | Keep images as
|
|
260
|
-
| `--no-extract-images` | | Alias for `--embed-images`
|
|
261
|
-
| `--
|
|
262
|
-
| `--
|
|
263
|
-
| `--
|
|
264
|
-
| `--
|
|
265
|
-
| `--
|
|
266
|
-
| `--
|
|
267
|
-
| `--
|
|
268
|
-
| `--extract-
|
|
269
|
-
| `--
|
|
270
|
-
| `--
|
|
271
|
-
| `--
|
|
272
|
-
| `--
|
|
273
|
-
| `--
|
|
254
|
+
| Option | Short | Description | Default |
|
|
255
|
+
| --------------------------- | ----- | ------------------------------------------------- | ----------------------------------- |
|
|
256
|
+
| `--format` | `-f` | Output format (see below) | `markdown` |
|
|
257
|
+
| `--output` | `-o` | Output file path. Use `-o -` for stdout | auto-derived from URL |
|
|
258
|
+
| `--data-dir` | | Base directory for auto-derived output paths | `./data/web-capture` |
|
|
259
|
+
| `--engine` | `-e` | Browser engine: `puppeteer`, `playwright` | `puppeteer` (or BROWSER_ENGINE env) |
|
|
260
|
+
| `--theme` | `-t` | Color scheme: `light`, `dark`, `no-preference` | browser default |
|
|
261
|
+
| `--width` | | Viewport width in pixels | 1280 |
|
|
262
|
+
| `--height` | | Viewport height in pixels | 800 |
|
|
263
|
+
| `--quality` | | JPEG quality 0-100 | 80 |
|
|
264
|
+
| `--fullPage` | | Capture full scrollable page | false |
|
|
265
|
+
| `--embed-images` | | Keep images inline as base64 (self-contained) | false |
|
|
266
|
+
| `--no-extract-images` | | Alias for `--embed-images` | false |
|
|
267
|
+
| `--extract-images[=DIR]` | | Extract images to `DIR/images/` + download remote | - |
|
|
268
|
+
| `--keep-original-links` | | Keep remote URLs as direct links (the default) | false |
|
|
269
|
+
| `--images-dir` | | Subdirectory name for extracted images | `images` |
|
|
270
|
+
| `--archive` | | Create archive: `zip`, `7z`, `tar.gz`, `tar` | - |
|
|
271
|
+
| `--document-format` | | Document format in archive: `markdown`, `html` | `markdown` |
|
|
272
|
+
| `--localImages` | | Download images locally in archive mode | true |
|
|
273
|
+
| `--extract-latex` | | Extract LaTeX formulas | true |
|
|
274
|
+
| `--no-extract-latex` | | Disable LaTeX extraction | - |
|
|
275
|
+
| `--extract-metadata` | | Extract article metadata | true |
|
|
276
|
+
| `--no-extract-metadata` | | Disable metadata extraction | - |
|
|
277
|
+
| `--post-process` | | Apply post-processing | true |
|
|
278
|
+
| `--no-post-process` | | Disable post-processing | - |
|
|
279
|
+
| `--detect-code-language` | | Detect code block languages | true |
|
|
280
|
+
| `--no-detect-code-language` | | Disable code language detection | - |
|
|
274
281
|
|
|
275
282
|
**Supported formats:**
|
|
276
283
|
|
package/bin/web-capture.js
CHANGED
|
@@ -154,7 +154,7 @@ const config = makeConfig({
|
|
|
154
154
|
.option('embedImages', {
|
|
155
155
|
type: 'boolean',
|
|
156
156
|
description:
|
|
157
|
-
'Keep images as
|
|
157
|
+
'Keep images inline as base64 data URIs, producing a single self-contained file (default: false). By default markdown keeps remote images as direct links; use --extract-images to save them as local files.',
|
|
158
158
|
default: getenv('WEB_CAPTURE_EMBED_IMAGES', false),
|
|
159
159
|
})
|
|
160
160
|
.option('imagesDir', {
|
|
@@ -163,6 +163,14 @@ const config = makeConfig({
|
|
|
163
163
|
'Directory name for extracted images, relative to output file (default: images)',
|
|
164
164
|
default: getenv('WEB_CAPTURE_IMAGES_DIR', 'images'),
|
|
165
165
|
})
|
|
166
|
+
.option('extractImages', {
|
|
167
|
+
type: 'string',
|
|
168
|
+
description:
|
|
169
|
+
'Extract images to local files under <dir>/images/ and rewrite the ' +
|
|
170
|
+
'markdown to reference them. Pass a directory, or use the flag with ' +
|
|
171
|
+
'no value to extract next to the output file. Downloads remote images too.',
|
|
172
|
+
default: getenv('WEB_CAPTURE_EXTRACT_IMAGES', undefined),
|
|
173
|
+
})
|
|
166
174
|
.option('dataDir', {
|
|
167
175
|
type: 'string',
|
|
168
176
|
description:
|
|
@@ -183,8 +191,9 @@ const config = makeConfig({
|
|
|
183
191
|
.option('keepOriginalLinks', {
|
|
184
192
|
type: 'boolean',
|
|
185
193
|
description:
|
|
186
|
-
'Keep
|
|
187
|
-
'
|
|
194
|
+
'Keep remote image URLs as direct links — the default markdown ' +
|
|
195
|
+
'behavior, kept as an explicit alias for back-compat. Base64 data ' +
|
|
196
|
+
'URIs are stripped (no original URL to restore).',
|
|
188
197
|
default: getenv('WEB_CAPTURE_KEEP_ORIGINAL_LINKS', false),
|
|
189
198
|
})
|
|
190
199
|
.option('dualTheme', {
|
|
@@ -337,6 +346,123 @@ function deriveOutputPath(absoluteUrl, ext, dataDir) {
|
|
|
337
346
|
return path.join(dir, `document.${ext}`);
|
|
338
347
|
}
|
|
339
348
|
|
|
349
|
+
/**
|
|
350
|
+
* Resolve the image mode for a markdown output destined for `outputDir`,
|
|
351
|
+
* mirroring the Rust CLI precedence: --embed-images > --extract-images >
|
|
352
|
+
* default (keep remote links, strip inline base64). See issue #112.
|
|
353
|
+
*
|
|
354
|
+
* @param {Object} options - Parsed CLI options
|
|
355
|
+
* @param {string} outputDir - Directory the markdown file is written to
|
|
356
|
+
* @returns {{mode: 'embed'|'extract'|'default', dir?: string, subdir: string}}
|
|
357
|
+
*/
|
|
358
|
+
function resolveImageMode(options, outputDir) {
|
|
359
|
+
const subdir = options.imagesDir || 'images';
|
|
360
|
+
if (options.embedImages) {
|
|
361
|
+
return { mode: 'embed', subdir };
|
|
362
|
+
}
|
|
363
|
+
const extract = options.extractImages;
|
|
364
|
+
// `--extract-images` with no value arrives as '' (or true); fall back to the
|
|
365
|
+
// output directory in that case, otherwise use the provided directory.
|
|
366
|
+
if (extract !== undefined && extract !== false && extract !== null) {
|
|
367
|
+
const dir = extract === '' || extract === true ? outputDir : extract;
|
|
368
|
+
return { mode: 'extract', dir, subdir };
|
|
369
|
+
}
|
|
370
|
+
return { mode: 'default', subdir };
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Download remote images that an `extract` mode localized, writing them under
|
|
375
|
+
* `imagesPath`. On any failure the original remote URL is restored so the
|
|
376
|
+
* markdown never points at a missing local file. Mirrors the Rust helper.
|
|
377
|
+
*
|
|
378
|
+
* @param {string} markdown - Markdown with localized remote references
|
|
379
|
+
* @param {Array<{url: string, filename: string}>} pending - Remote images to fetch
|
|
380
|
+
* @param {string} imagesPath - Absolute directory to write images into
|
|
381
|
+
* @param {string} subdir - Images subdirectory name (for the local reference)
|
|
382
|
+
* @returns {Promise<string>} The (possibly URL-restored) markdown
|
|
383
|
+
*/
|
|
384
|
+
async function downloadPendingRemote(markdown, pending, imagesPath, subdir) {
|
|
385
|
+
let result = markdown;
|
|
386
|
+
let downloaded = 0;
|
|
387
|
+
for (const img of pending) {
|
|
388
|
+
const localRef = `${subdir}/${img.filename}`;
|
|
389
|
+
let buffer = null;
|
|
390
|
+
try {
|
|
391
|
+
const resp = await fetch(img.url);
|
|
392
|
+
if (resp.ok) {
|
|
393
|
+
buffer = Buffer.from(await resp.arrayBuffer());
|
|
394
|
+
}
|
|
395
|
+
} catch {
|
|
396
|
+
buffer = null;
|
|
397
|
+
}
|
|
398
|
+
let saved = false;
|
|
399
|
+
if (buffer) {
|
|
400
|
+
try {
|
|
401
|
+
fs.mkdirSync(imagesPath, { recursive: true });
|
|
402
|
+
fs.writeFileSync(path.join(imagesPath, img.filename), buffer);
|
|
403
|
+
saved = true;
|
|
404
|
+
} catch {
|
|
405
|
+
saved = false;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
if (saved) {
|
|
409
|
+
downloaded++;
|
|
410
|
+
} else {
|
|
411
|
+
// Restore the original URL so the reference is not broken.
|
|
412
|
+
result = result.split(localRef).join(img.url);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
if (downloaded > 0) {
|
|
416
|
+
console.error(`Downloaded ${downloaded} remote image(s) to ${subdir}/`);
|
|
417
|
+
}
|
|
418
|
+
return result;
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
/**
|
|
422
|
+
* Apply the resolved image mode to markdown destined for `outputPath`,
|
|
423
|
+
* downloading any remote images an extract mode localized. The single
|
|
424
|
+
* image-handling chokepoint for the JS CLI — every markdown output path routes
|
|
425
|
+
* through this so flags behave uniformly regardless of capture method (#112).
|
|
426
|
+
*
|
|
427
|
+
* @param {string} markdown - Markdown content
|
|
428
|
+
* @param {Object} options - Parsed CLI options
|
|
429
|
+
* @param {string} outputPath - File the markdown will be written to
|
|
430
|
+
* @param {string} label - Human label for log messages
|
|
431
|
+
* @returns {Promise<string>} The rewritten markdown
|
|
432
|
+
*/
|
|
433
|
+
async function processOutputMarkdown(markdown, options, outputPath, label) {
|
|
434
|
+
const { applyImageMode } = await import('../src/extract-images.js');
|
|
435
|
+
const outputDir = path.dirname(outputPath);
|
|
436
|
+
const resolved = resolveImageMode(options, outputDir);
|
|
437
|
+
const result = await applyImageMode(markdown, {
|
|
438
|
+
mode: resolved.mode,
|
|
439
|
+
dir: resolved.dir,
|
|
440
|
+
subdir: resolved.subdir,
|
|
441
|
+
});
|
|
442
|
+
let md = result.markdown;
|
|
443
|
+
if (result.extracted > 0) {
|
|
444
|
+
console.error(
|
|
445
|
+
`Extracted ${result.extracted} images to ${resolved.subdir}/ (${label})`
|
|
446
|
+
);
|
|
447
|
+
}
|
|
448
|
+
if (result.stripped > 0) {
|
|
449
|
+
console.error(
|
|
450
|
+
`Stripped ${result.stripped} inline base64 image(s) (${label}); ` +
|
|
451
|
+
'use --extract-images to save files or --embed-images to keep them inline'
|
|
452
|
+
);
|
|
453
|
+
}
|
|
454
|
+
if (resolved.mode === 'extract' && result.pendingRemote.length > 0) {
|
|
455
|
+
const imagesPath = path.resolve(resolved.dir, resolved.subdir);
|
|
456
|
+
md = await downloadPendingRemote(
|
|
457
|
+
md,
|
|
458
|
+
result.pendingRemote,
|
|
459
|
+
imagesPath,
|
|
460
|
+
resolved.subdir
|
|
461
|
+
);
|
|
462
|
+
}
|
|
463
|
+
return md;
|
|
464
|
+
}
|
|
465
|
+
|
|
340
466
|
function archiveExtension(archiveFormat) {
|
|
341
467
|
if (archiveFormat === 'tar.gz' || archiveFormat === 'gz') {
|
|
342
468
|
return 'tar.gz';
|
|
@@ -426,7 +552,8 @@ async function captureUrl(url, options) {
|
|
|
426
552
|
// localImages is used by archive format
|
|
427
553
|
// eslint-disable-next-line no-unused-vars
|
|
428
554
|
localImages,
|
|
429
|
-
embedImages
|
|
555
|
+
// imagesDir/embedImages/extractImages are read via `options` in the image
|
|
556
|
+
// mode chokepoint (resolveImageMode); destructured only for documentation.
|
|
430
557
|
// eslint-disable-next-line no-unused-vars
|
|
431
558
|
imagesDir,
|
|
432
559
|
dataDir,
|
|
@@ -462,7 +589,6 @@ async function captureUrl(url, options) {
|
|
|
462
589
|
captureGoogleDocWithBrowserOrFallback,
|
|
463
590
|
selectGoogleDocsCaptureMethod,
|
|
464
591
|
} = await import('../src/gdocs.js');
|
|
465
|
-
const { stripBase64Images } = await import('../src/extract-images.js');
|
|
466
592
|
|
|
467
593
|
const normalizedFormat = format.toLowerCase();
|
|
468
594
|
log.debug(() => ({
|
|
@@ -649,8 +775,6 @@ async function captureUrl(url, options) {
|
|
|
649
775
|
}
|
|
650
776
|
} else {
|
|
651
777
|
try {
|
|
652
|
-
const { extractAndSaveImages } =
|
|
653
|
-
await import('../src/extract-images.js');
|
|
654
778
|
if (normalizedFormat === 'archive' || normalizedFormat === 'zip') {
|
|
655
779
|
const { fetchGoogleDocAsArchive } = await import('../src/gdocs.js');
|
|
656
780
|
const { default: archiver } = await import('archiver');
|
|
@@ -721,36 +845,20 @@ async function captureUrl(url, options) {
|
|
|
721
845
|
apiToken,
|
|
722
846
|
log,
|
|
723
847
|
});
|
|
724
|
-
|
|
848
|
+
const { markdown } = result;
|
|
725
849
|
const output =
|
|
726
850
|
explicitOutput === '-'
|
|
727
851
|
? null
|
|
728
852
|
: explicitOutput || deriveOutputPath(absoluteUrl, 'md', dataDir);
|
|
729
853
|
if (output) {
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
console.error(
|
|
737
|
-
`Stripped ${strip.stripped} base64 images (keeping original links)`
|
|
738
|
-
);
|
|
739
|
-
}
|
|
740
|
-
} else {
|
|
741
|
-
const outputDir = path.dirname(output);
|
|
742
|
-
const extracted = extractAndSaveImages(markdown, outputDir, {
|
|
743
|
-
imagesDir: options.imagesDir || 'images',
|
|
744
|
-
});
|
|
745
|
-
if (extracted.extracted > 0) {
|
|
746
|
-
markdown = extracted.markdown;
|
|
747
|
-
console.error(
|
|
748
|
-
`Extracted ${extracted.extracted} images to ${options.imagesDir || 'images'}/`
|
|
749
|
-
);
|
|
750
|
-
}
|
|
751
|
-
}
|
|
854
|
+
const finalMarkdown = await processOutputMarkdown(
|
|
855
|
+
markdown,
|
|
856
|
+
options,
|
|
857
|
+
output,
|
|
858
|
+
'Google Doc Markdown'
|
|
859
|
+
);
|
|
752
860
|
fs.mkdirSync(path.dirname(output), { recursive: true });
|
|
753
|
-
fs.writeFileSync(output,
|
|
861
|
+
fs.writeFileSync(output, finalMarkdown, 'utf-8');
|
|
754
862
|
console.error(`Google Doc Markdown saved to: ${output}`);
|
|
755
863
|
} else {
|
|
756
864
|
process.stdout.write(markdown);
|
|
@@ -999,7 +1107,6 @@ async function captureUrl(url, options) {
|
|
|
999
1107
|
// Markdown format — enhanced conversion is now the default
|
|
1000
1108
|
const html = await fetchHtml(absoluteUrl);
|
|
1001
1109
|
const { convertHtmlToMarkdownEnhanced } = await import('../src/lib.js');
|
|
1002
|
-
const { extractAndSaveImages } = await import('../src/extract-images.js');
|
|
1003
1110
|
const result = convertHtmlToMarkdownEnhanced(html, absoluteUrl, {
|
|
1004
1111
|
extractLatex: options.extractLatex,
|
|
1005
1112
|
extractMetadata: options.extractMetadata,
|
|
@@ -1008,40 +1115,21 @@ async function captureUrl(url, options) {
|
|
|
1008
1115
|
contentSelector: options.contentSelector,
|
|
1009
1116
|
bodySelector: options.bodySelector,
|
|
1010
1117
|
});
|
|
1011
|
-
|
|
1118
|
+
const markdown = result.markdown;
|
|
1012
1119
|
|
|
1013
1120
|
const output =
|
|
1014
1121
|
explicitOutput === '-'
|
|
1015
1122
|
? null
|
|
1016
1123
|
: explicitOutput || deriveOutputPath(absoluteUrl, 'md', dataDir);
|
|
1017
1124
|
if (output) {
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
console.error(
|
|
1025
|
-
`Stripped ${strip.stripped} base64 images (keeping original links)`
|
|
1026
|
-
);
|
|
1027
|
-
}
|
|
1028
|
-
} else {
|
|
1029
|
-
const outputDir = path.dirname(output);
|
|
1030
|
-
const extracted = extractAndSaveImages(markdown, outputDir, {
|
|
1031
|
-
imagesDir: options.imagesDir || 'images',
|
|
1032
|
-
});
|
|
1033
|
-
if (extracted.extracted > 0) {
|
|
1034
|
-
markdown = extracted.markdown;
|
|
1035
|
-
console.error(
|
|
1036
|
-
`Extracted ${extracted.extracted} images to ${options.imagesDir || 'images'}/`
|
|
1037
|
-
);
|
|
1038
|
-
}
|
|
1039
|
-
}
|
|
1040
|
-
}
|
|
1041
|
-
|
|
1042
|
-
if (output) {
|
|
1125
|
+
const finalMarkdown = await processOutputMarkdown(
|
|
1126
|
+
markdown,
|
|
1127
|
+
options,
|
|
1128
|
+
output,
|
|
1129
|
+
'Markdown'
|
|
1130
|
+
);
|
|
1043
1131
|
fs.mkdirSync(path.dirname(output), { recursive: true });
|
|
1044
|
-
fs.writeFileSync(output,
|
|
1132
|
+
fs.writeFileSync(output, finalMarkdown, 'utf-8');
|
|
1045
1133
|
console.error(`Markdown saved to: ${output}`);
|
|
1046
1134
|
} else {
|
|
1047
1135
|
process.stdout.write(markdown);
|
|
@@ -1192,6 +1280,7 @@ async function main() {
|
|
|
1192
1280
|
dualTheme: config.dualTheme,
|
|
1193
1281
|
apiToken: config.apiToken,
|
|
1194
1282
|
embedImages: config.embedImages,
|
|
1283
|
+
extractImages: config.extractImages,
|
|
1195
1284
|
imagesDir: config.imagesDir,
|
|
1196
1285
|
dataDir: config.dataDir,
|
|
1197
1286
|
archiveFormat: config.archiveFormat,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-assistant/web-capture",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.8.0",
|
|
4
4
|
"description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -65,7 +65,8 @@
|
|
|
65
65
|
"markdown-it": "^14.1.1",
|
|
66
66
|
"nock": "^13.5.4",
|
|
67
67
|
"prettier": "^3.6.2",
|
|
68
|
-
"supertest": "^6.3.4"
|
|
68
|
+
"supertest": "^6.3.4",
|
|
69
|
+
"unzipper": "^0.12.3"
|
|
69
70
|
},
|
|
70
71
|
"type": "module",
|
|
71
72
|
"keywords": [
|
package/src/archive.js
CHANGED
|
@@ -170,14 +170,17 @@ export async function archiveHandler(req, res) {
|
|
|
170
170
|
}
|
|
171
171
|
|
|
172
172
|
if (!embedImages && !keepOriginalLinks) {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
archive.append(img.buffer, { name: `images/${img.filename}` });
|
|
177
|
-
}
|
|
173
|
+
appendMarkdownAndImages(archive, markdown);
|
|
174
|
+
} else {
|
|
175
|
+
archive.append(markdown, { name: 'document.md' });
|
|
178
176
|
}
|
|
179
177
|
|
|
180
|
-
|
|
178
|
+
// document.html — the source the markdown was derived from, for
|
|
179
|
+
// reference only, so the default archive layout (document.md +
|
|
180
|
+
// document.html + images/) is identical across capture paths (issue #113).
|
|
181
|
+
archive.append(prettyPrintHtml(convertRelativeUrls(html, absoluteUrl)), {
|
|
182
|
+
name: 'document.html',
|
|
183
|
+
});
|
|
181
184
|
}
|
|
182
185
|
|
|
183
186
|
// Download and add images if local mode
|
|
@@ -207,6 +210,57 @@ export async function archiveHandler(req, res) {
|
|
|
207
210
|
}
|
|
208
211
|
}
|
|
209
212
|
|
|
213
|
+
/**
|
|
214
|
+
* Extract base64 images from `markdown` into the archive's `images/` folder and
|
|
215
|
+
* append the rewritten markdown as `document.md`. Shared by the archive
|
|
216
|
+
* endpoint and {@link buildArchiveFromHtml} so the default layout stays
|
|
217
|
+
* identical across capture paths.
|
|
218
|
+
*
|
|
219
|
+
* @param {import('archiver').Archiver} archive - The archive being built
|
|
220
|
+
* @param {string} markdown - Markdown content (may contain base64 data URIs)
|
|
221
|
+
*/
|
|
222
|
+
function appendMarkdownAndImages(archive, markdown) {
|
|
223
|
+
const { markdown: rewritten, images } = extractBase64ToBuffers(
|
|
224
|
+
markdown,
|
|
225
|
+
'images'
|
|
226
|
+
);
|
|
227
|
+
for (const img of images) {
|
|
228
|
+
archive.append(img.buffer, { name: `images/${img.filename}` });
|
|
229
|
+
}
|
|
230
|
+
archive.append(rewritten, { name: 'document.md' });
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Build a self-contained ZIP archive (as a Buffer) from raw HTML, matching the
|
|
235
|
+
* default `--format archive` layout contract (issue #113):
|
|
236
|
+
*
|
|
237
|
+
* - `document.md` — markdown referencing images by relative `images/` path
|
|
238
|
+
* - `document.html` — the source HTML the markdown was derived from (reference)
|
|
239
|
+
* - `images/` — every inline base64 image as a separate file
|
|
240
|
+
*
|
|
241
|
+
* @param {string} html - Source HTML to convert
|
|
242
|
+
* @param {string} baseUrl - Base URL for resolving relative links
|
|
243
|
+
* @returns {Promise<Buffer>} the finalized ZIP archive bytes
|
|
244
|
+
*/
|
|
245
|
+
export async function buildArchiveFromHtml(html, baseUrl) {
|
|
246
|
+
const archive = archiver('zip', { zlib: { level: 9 } });
|
|
247
|
+
const chunks = [];
|
|
248
|
+
archive.on('data', (chunk) => chunks.push(chunk));
|
|
249
|
+
const finished = new Promise((resolve, reject) => {
|
|
250
|
+
archive.on('end', resolve);
|
|
251
|
+
archive.on('error', reject);
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
appendMarkdownAndImages(archive, convertHtmlToMarkdown(html, baseUrl));
|
|
255
|
+
archive.append(prettyPrintHtml(convertRelativeUrls(html, baseUrl)), {
|
|
256
|
+
name: 'document.html',
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
await archive.finalize();
|
|
260
|
+
await finished;
|
|
261
|
+
return Buffer.concat(chunks);
|
|
262
|
+
}
|
|
263
|
+
|
|
210
264
|
function guessImageExtension(url) {
|
|
211
265
|
const pathname = new URL(url).pathname.toLowerCase();
|
|
212
266
|
if (pathname.endsWith('.jpg') || pathname.endsWith('.jpeg')) {
|
package/src/extract-images.js
CHANGED
|
@@ -128,3 +128,93 @@ export function stripBase64Images(markdown) {
|
|
|
128
128
|
export function hasBase64Images(markdown) {
|
|
129
129
|
return BASE64_IMAGE_REGEX.test(markdown);
|
|
130
130
|
}
|
|
131
|
+
|
|
132
|
+
// Matches a markdown image whose source is a remote http(s) URL. A trailing
|
|
133
|
+
// markdown title attribute (e.g. ``) is matched but excluded
|
|
134
|
+
// from the captured URL.
|
|
135
|
+
const REMOTE_IMAGE_REGEX =
|
|
136
|
+
/!\[([^\]]*)\]\((https?:\/\/[^)\s]+)(?:\s+"[^"]*")?\)/g;
|
|
137
|
+
|
|
138
|
+
function remoteImageExtension(url) {
|
|
139
|
+
const cleaned = url.split(/[?#]/)[0];
|
|
140
|
+
const ext = (cleaned.split('.').pop() || '').toLowerCase();
|
|
141
|
+
if (ext === 'jpg' || ext === 'jpeg') {
|
|
142
|
+
return 'jpg';
|
|
143
|
+
}
|
|
144
|
+
if (['gif', 'webp', 'svg'].includes(ext)) {
|
|
145
|
+
return ext;
|
|
146
|
+
}
|
|
147
|
+
return 'png';
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Apply an image mode to markdown — the single image-handling chokepoint.
|
|
152
|
+
*
|
|
153
|
+
* Every CLI/server capture path routes through this function so the same flag
|
|
154
|
+
* produces the same result regardless of capture method (browser vs API). See
|
|
155
|
+
* issue #112.
|
|
156
|
+
*
|
|
157
|
+
* Modes:
|
|
158
|
+
* - `'default'`: keep remote URLs as direct links; strip inline base64 data
|
|
159
|
+
* URIs (no remote URL to restore) to a visible placeholder. No `images/`
|
|
160
|
+
* folder, no silently-kept multi-megabyte base64 blob.
|
|
161
|
+
* - `'embed'`: keep base64 inline (single self-contained file).
|
|
162
|
+
* - `'extract'`: extract base64 to files under `dir/subdir` and rewrite remote
|
|
163
|
+
* references to the same local `subdir/` paths. The remote bytes are
|
|
164
|
+
* downloaded by the caller — see `pendingRemote` in the result.
|
|
165
|
+
*
|
|
166
|
+
* @param {string} markdown - Markdown content
|
|
167
|
+
* @param {Object} [options]
|
|
168
|
+
* @param {'default'|'embed'|'extract'} [options.mode='default']
|
|
169
|
+
* @param {string} [options.dir] - Output directory (required for 'extract')
|
|
170
|
+
* @param {string} [options.subdir='images'] - Images subdirectory name
|
|
171
|
+
* @returns {Promise<{markdown: string, extracted: number, stripped: number, pendingRemote: Array<{url: string, filename: string}>}>}
|
|
172
|
+
*/
|
|
173
|
+
// Async by contract: this is the single chokepoint every capture path awaits,
|
|
174
|
+
// and remote localization is fulfilled asynchronously by the caller. The body
|
|
175
|
+
// itself is currently synchronous.
|
|
176
|
+
// eslint-disable-next-line require-await
|
|
177
|
+
export async function applyImageMode(markdown, options = {}) {
|
|
178
|
+
const { mode = 'default', dir, subdir = 'images' } = options;
|
|
179
|
+
|
|
180
|
+
if (mode === 'embed') {
|
|
181
|
+
return { markdown, extracted: 0, stripped: 0, pendingRemote: [] };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
if (mode === 'extract') {
|
|
185
|
+
if (!dir) {
|
|
186
|
+
throw new Error("applyImageMode: 'extract' mode requires an output dir");
|
|
187
|
+
}
|
|
188
|
+
// 1. Extract inline base64 images to files.
|
|
189
|
+
const extracted = extractAndSaveImages(markdown, dir, {
|
|
190
|
+
imagesDir: subdir,
|
|
191
|
+
});
|
|
192
|
+
// 2. Plan localization of remote image references to the same folder.
|
|
193
|
+
const pendingRemote = [];
|
|
194
|
+
let index = 0;
|
|
195
|
+
const localized = extracted.markdown.replace(
|
|
196
|
+
REMOTE_IMAGE_REGEX,
|
|
197
|
+
(_match, altText, url) => {
|
|
198
|
+
index++;
|
|
199
|
+
const filename = `image-${String(index).padStart(2, '0')}.${remoteImageExtension(url)}`;
|
|
200
|
+
pendingRemote.push({ url, filename });
|
|
201
|
+
return ``;
|
|
202
|
+
}
|
|
203
|
+
);
|
|
204
|
+
return {
|
|
205
|
+
markdown: localized,
|
|
206
|
+
extracted: extracted.extracted,
|
|
207
|
+
stripped: 0,
|
|
208
|
+
pendingRemote,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// 'default'
|
|
213
|
+
const strip = stripBase64Images(markdown);
|
|
214
|
+
return {
|
|
215
|
+
markdown: strip.markdown,
|
|
216
|
+
extracted: 0,
|
|
217
|
+
stripped: strip.stripped,
|
|
218
|
+
pendingRemote: [],
|
|
219
|
+
};
|
|
220
|
+
}
|
package/src/markdown.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { fetchHtml, convertHtmlToMarkdownEnhanced } from './lib.js';
|
|
2
|
-
import {
|
|
2
|
+
import { applyImageMode } from './extract-images.js';
|
|
3
3
|
|
|
4
4
|
export async function markdownHandler(req, res) {
|
|
5
5
|
const url = req.query.url;
|
|
@@ -9,15 +9,17 @@ export async function markdownHandler(req, res) {
|
|
|
9
9
|
const embedImages = req.query.embedImages === 'true';
|
|
10
10
|
try {
|
|
11
11
|
const html = await fetchHtml(url);
|
|
12
|
-
|
|
12
|
+
const { markdown } = convertHtmlToMarkdownEnhanced(html, url, {
|
|
13
13
|
contentSelector: req.query.contentSelector,
|
|
14
14
|
bodySelector: req.query.bodySelector,
|
|
15
15
|
});
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
16
|
+
// Route through the single image-mode chokepoint so the server honors the
|
|
17
|
+
// same contract as the CLI: default keeps remote links and strips inline
|
|
18
|
+
// base64; ?embedImages=true keeps base64 inline. See issue #112.
|
|
19
|
+
const result = await applyImageMode(markdown, {
|
|
20
|
+
mode: embedImages ? 'embed' : 'default',
|
|
21
|
+
});
|
|
22
|
+
res.type('text/markdown').send(result.markdown);
|
|
21
23
|
} catch (err) {
|
|
22
24
|
console.error(err);
|
|
23
25
|
res.status(500).send('Error converting to Markdown');
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
import { jest } from '@jest/globals';
|
|
10
10
|
import request from 'supertest';
|
|
11
11
|
import nock from 'nock';
|
|
12
|
+
import unzipper from 'unzipper';
|
|
12
13
|
import { app } from '../../src/index.js';
|
|
13
14
|
|
|
14
15
|
jest.setTimeout(60000);
|
|
@@ -95,6 +96,14 @@ describe('API Endpoint Tests', () => {
|
|
|
95
96
|
// ZIP signature: PK (0x50 0x4B)
|
|
96
97
|
expect(res.body[0]).toBe(0x50);
|
|
97
98
|
expect(res.body[1]).toBe(0x4b);
|
|
99
|
+
|
|
100
|
+
// Default archive layout contract (issue #113): the markdown archive must
|
|
101
|
+
// bundle document.md AND document.html for reference.
|
|
102
|
+
const dir = await unzipper.Open.buffer(res.body);
|
|
103
|
+
const names = dir.files.map((f) => f.path);
|
|
104
|
+
expect(names).toEqual(
|
|
105
|
+
expect.arrayContaining(['document.md', 'document.html'])
|
|
106
|
+
);
|
|
98
107
|
});
|
|
99
108
|
|
|
100
109
|
it('returns a ZIP archive with HTML format', async () => {
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pin the default `--format archive` layout contract (issue #113):
|
|
3
|
+
* the zip contains exactly `document.md` + `document.html` + `images/`.
|
|
4
|
+
*/
|
|
5
|
+
import unzipper from 'unzipper';
|
|
6
|
+
import { buildArchiveFromHtml } from '../../src/archive.js';
|
|
7
|
+
|
|
8
|
+
const TINY_PNG =
|
|
9
|
+
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
|
|
10
|
+
const FIXTURE_HTML = `<!doctype html><html><body>
|
|
11
|
+
<h1>Hi</h1><p>Para 1.</p>
|
|
12
|
+
<p><img alt="" src="data:image/png;base64,${TINY_PNG}"></p>
|
|
13
|
+
<p>Para 2.</p>
|
|
14
|
+
</body></html>`;
|
|
15
|
+
|
|
16
|
+
it('default archive contains document.md, document.html, and images/ folder', async () => {
|
|
17
|
+
const buf = await buildArchiveFromHtml(
|
|
18
|
+
FIXTURE_HTML,
|
|
19
|
+
'https://example.invalid/'
|
|
20
|
+
);
|
|
21
|
+
const dir = await unzipper.Open.buffer(buf);
|
|
22
|
+
const names = dir.files.map((f) => f.path);
|
|
23
|
+
|
|
24
|
+
expect(names).toEqual(
|
|
25
|
+
expect.arrayContaining(['document.md', 'document.html'])
|
|
26
|
+
);
|
|
27
|
+
expect(names.some((n) => n.startsWith('images/'))).toBe(true);
|
|
28
|
+
|
|
29
|
+
const mdEntry = dir.files.find((f) => f.path === 'document.md');
|
|
30
|
+
const md = (await mdEntry.buffer()).toString();
|
|
31
|
+
expect(md).toMatch(/images\//);
|
|
32
|
+
expect(md).not.toMatch(/data:image/);
|
|
33
|
+
|
|
34
|
+
const htmlEntry = dir.files.find((f) => f.path === 'document.html');
|
|
35
|
+
const html = (await htmlEntry.buffer()).toString();
|
|
36
|
+
expect(html).toMatch(/<h1>/);
|
|
37
|
+
});
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { applyImageMode } from '../../src/extract-images.js';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import os from 'os';
|
|
5
|
+
|
|
6
|
+
const TINY_PNG =
|
|
7
|
+
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
|
|
8
|
+
const REMOTE = `Hi.\n\n\n\nBye.\n`;
|
|
9
|
+
const BASE64 = `Hi.\n\n\n\nBye.\n`;
|
|
10
|
+
|
|
11
|
+
describe('image-mode contract', () => {
|
|
12
|
+
let tmp;
|
|
13
|
+
beforeEach(() => {
|
|
14
|
+
tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'imode-'));
|
|
15
|
+
});
|
|
16
|
+
afterEach(() => fs.rmSync(tmp, { recursive: true, force: true }));
|
|
17
|
+
|
|
18
|
+
it('default mode keeps remote URLs', async () => {
|
|
19
|
+
const r = await applyImageMode(REMOTE, { mode: 'default' });
|
|
20
|
+
expect(r.markdown).toContain('https://example.invalid/foo.png');
|
|
21
|
+
expect(r.markdown).not.toMatch(/images\//);
|
|
22
|
+
expect(r.markdown).not.toMatch(/data:image/);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('--extract-images writes files', async () => {
|
|
26
|
+
const r = await applyImageMode(BASE64, {
|
|
27
|
+
mode: 'extract',
|
|
28
|
+
dir: tmp,
|
|
29
|
+
subdir: 'images',
|
|
30
|
+
});
|
|
31
|
+
expect(r.markdown).toMatch(/images\/image-/);
|
|
32
|
+
expect(fs.readdirSync(path.join(tmp, 'images'))).toHaveLength(1);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it('--embed-images keeps base64', async () => {
|
|
36
|
+
const r = await applyImageMode(BASE64, { mode: 'embed' });
|
|
37
|
+
expect(r.markdown).toContain('data:image');
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it('flags have observable effect on every input', async () => {
|
|
41
|
+
for (const src of [REMOTE, BASE64]) {
|
|
42
|
+
const a = await applyImageMode(src, { mode: 'default' });
|
|
43
|
+
const b = await applyImageMode(src, { mode: 'embed' });
|
|
44
|
+
const c = await applyImageMode(src, {
|
|
45
|
+
mode: 'extract',
|
|
46
|
+
dir: tmp,
|
|
47
|
+
subdir: 'images',
|
|
48
|
+
});
|
|
49
|
+
expect(
|
|
50
|
+
new Set([a.markdown, b.markdown, c.markdown]).size
|
|
51
|
+
).toBeGreaterThan(1);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
});
|